package com.neo.word.count

/**
  * Created by hadoop on 12/11/16.
  */
import org.apache.spark.{SparkConf, SparkContext}
import org.slf4j.LoggerFactory
import com.typesafe.scalalogging.slf4j.Logger


object WordCount {
  def main(args: Array[String]){
    val logger = Logger(LoggerFactory.getLogger("WordCount"))
    println("***********************")
    logger.error("Hello world0")
    println("***********************")
    val logFile = "hdfs://c64-dt:8020/user/test/wordcount/testdata/"
    logger.error("Hello world1")

    val conf  = new SparkConf().setAppName("Sample Appliction").setMaster("local[8]")
    //val conf  = new SparkConf().setAppName("Sample Appliction").setMaster("local")
    //val conf  = new SparkConf().setAppName("Sample Appliction")
    logger.error("Hello world2")
    val sc = new SparkContext(conf)
    logger.error("Hello world3")
    val logData = sc.textFile(logFile,2).cache()
    logger.error("Hello world4")
    val numAs = logData.filter(
      line => line.contains("a")).count()
    logger.error("Hello world5")
    val numBs = logData.filter(line => line.contains("b")).count()
    println("***********************")
    logger.error("*******************")
    logger.error("*******************")
    logger.error("*******************")
    logger.error("*******************")
    logger.error("*******************")
    logger.error("*******************")
    logger.error("*******************")
    logger.error("*******************")

    println("Lines with a : %s , Lines with b: %s".format(numAs,numBs))

  }
}
